{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Please input your directory for the top level folder\n",
    "folder name : SUBMISSION MODEL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "dir_ = \"INPUT-PROJECT-DIRECTORY/submission_model/\"  # input only here"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### setting other directory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "raw_data_dir = dir_ + \"2. data/\"\n",
    "processed_data_dir = dir_ + \"2. data/processed/\"\n",
    "log_dir = dir_ + \"4. logs/\"\n",
    "model_dir = dir_ + \"5. models/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "####################################################################################\n",
    "########################### 1-1. recursive model by store ##########################\n",
    "####################################################################################"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "ver, KKK = \"priv\", 0\n",
    "STORES_IDS = [\n",
    "    \"CA_1\",\n",
    "    \"CA_2\",\n",
    "    \"CA_3\",\n",
    "    \"CA_4\",\n",
    "    \"TX_1\",\n",
    "    \"TX_2\",\n",
    "    \"TX_3\",\n",
    "    \"WI_1\",\n",
    "    \"WI_2\",\n",
    "    \"WI_3\",\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os, sys, gc, time, warnings, pickle, psutil, random\n",
    "\n",
    "from multiprocessing import Pool\n",
    "\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "########################### Helpers\n",
    "#################################################################################\n",
    "## Seeder\n",
    "def seed_everything(seed=0):\n",
    "    random.seed(seed)\n",
    "    np.random.seed(seed)\n",
    "\n",
    "\n",
    "## Multiprocess Runs\n",
    "def df_parallelize_run(func, t_split):\n",
    "    num_cores = np.min([N_CORES, len(t_split)])\n",
    "    pool = Pool(num_cores)\n",
    "    df = pd.concat(pool.map(func, t_split), axis=1)\n",
    "    pool.close()\n",
    "    pool.join()\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "########################### Helper to load data by store ID\n",
    "#################################################################################\n",
    "# Read data\n",
    "def get_data_by_store(store):\n",
    "\n",
    "    # Read and contact basic feature\n",
    "    df = pd.concat(\n",
    "        [\n",
    "            pd.read_pickle(BASE),\n",
    "            pd.read_pickle(PRICE).iloc[:, 2:],\n",
    "            pd.read_pickle(CALENDAR).iloc[:, 2:],\n",
    "        ],\n",
    "        axis=1,\n",
    "    )\n",
    "\n",
    "    df = df[df[\"d\"] >= START_TRAIN]\n",
    "\n",
    "    df = df[df[\"store_id\"] == store]\n",
    "\n",
    "    df2 = pd.read_pickle(MEAN_ENC)[mean_features]\n",
    "    df2 = df2[df2.index.isin(df.index)]\n",
    "\n",
    "    df3 = pd.read_pickle(LAGS).iloc[:, 3:]\n",
    "    df3 = df3[df3.index.isin(df.index)]\n",
    "\n",
    "    df = pd.concat([df, df2], axis=1)\n",
    "    del df2\n",
    "\n",
    "    df = pd.concat([df, df3], axis=1)\n",
    "    del df3\n",
    "\n",
    "    features = [col for col in list(df) if col not in remove_features]\n",
    "    df = df[[\"id\", \"d\", TARGET] + features]\n",
    "\n",
    "    df = df.reset_index(drop=True)\n",
    "\n",
    "    return df, features\n",
    "\n",
    "\n",
    "# Recombine Test set after training\n",
    "def get_base_test():\n",
    "    base_test = pd.DataFrame()\n",
    "\n",
    "    for store_id in STORES_IDS:\n",
    "        temp_df = pd.read_pickle(\n",
    "            processed_data_dir + \"test_\" + store_id + \".pkl\"\n",
    "        )\n",
    "        temp_df[\"store_id\"] = store_id\n",
    "        base_test = pd.concat([base_test, temp_df]).reset_index(drop=True)\n",
    "\n",
    "    return base_test\n",
    "\n",
    "\n",
    "########################### Helper to make dynamic rolling lags\n",
    "#################################################################################\n",
    "def make_lag(LAG_DAY):\n",
    "    lag_df = base_test[[\"id\", \"d\", TARGET]]\n",
    "    col_name = \"sales_lag_\" + str(LAG_DAY)\n",
    "    lag_df[col_name] = (\n",
    "        lag_df.groupby([\"id\"])[TARGET]\n",
    "        .transform(lambda x: x.shift(LAG_DAY))\n",
    "        .astype(np.float16)\n",
    "    )\n",
    "    return lag_df[[col_name]]\n",
    "\n",
    "\n",
    "def make_lag_roll(LAG_DAY):\n",
    "    shift_day = LAG_DAY[0]\n",
    "    roll_wind = LAG_DAY[1]\n",
    "    lag_df = base_test[[\"id\", \"d\", TARGET]]\n",
    "    col_name = \"rolling_mean_tmp_\" + str(shift_day) + \"_\" + str(roll_wind)\n",
    "    lag_df[col_name] = lag_df.groupby([\"id\"])[TARGET].transform(\n",
    "        lambda x: x.shift(shift_day).rolling(roll_wind).mean()\n",
    "    )\n",
    "    return lag_df[[col_name]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "########################### Model params\n",
    "#################################################################################\n",
    "import lightgbm as lgb\n",
    "\n",
    "lgb_params = {\n",
    "    \"boosting_type\": \"gbdt\",\n",
    "    \"objective\": \"tweedie\",\n",
    "    \"tweedie_variance_power\": 1.1,\n",
    "    \"metric\": \"rmse\",\n",
    "    \"subsample\": 0.5,\n",
    "    \"subsample_freq\": 1,\n",
    "    \"learning_rate\": 0.015,\n",
    "    \"num_leaves\": 2**11 - 1,\n",
    "    \"min_data_in_leaf\": 2**12 - 1,\n",
    "    \"feature_fraction\": 0.5,\n",
    "    \"max_bin\": 100,\n",
    "    \"n_estimators\": 3000,\n",
    "    \"boost_from_average\": False,\n",
    "    \"verbose\": -1,\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "########################### Vars\n",
    "#################################################################################\n",
    "VER = 1\n",
    "SEED = 42\n",
    "seed_everything(SEED)\n",
    "lgb_params[\"seed\"] = SEED\n",
    "N_CORES = psutil.cpu_count()\n",
    "\n",
    "\n",
    "# LIMITS and const\n",
    "TARGET = \"sales\"\n",
    "START_TRAIN = 0\n",
    "END_TRAIN = 1941 - 28 * KKK\n",
    "P_HORIZON = 28\n",
    "USE_AUX = False\n",
    "\n",
    "remove_features = [\n",
    "    \"id\",\n",
    "    \"state_id\",\n",
    "    \"store_id\",\n",
    "    \"date\",\n",
    "    \"wm_yr_wk\",\n",
    "    \"d\",\n",
    "    TARGET,\n",
    "]\n",
    "mean_features = [\n",
    "    \"enc_cat_id_mean\",\n",
    "    \"enc_cat_id_std\",\n",
    "    \"enc_dept_id_mean\",\n",
    "    \"enc_dept_id_std\",\n",
    "    \"enc_item_id_mean\",\n",
    "    \"enc_item_id_std\",\n",
    "]\n",
    "\n",
    "ORIGINAL = raw_data_dir\n",
    "BASE = processed_data_dir + \"grid_part_1.pkl\"\n",
    "PRICE = processed_data_dir + \"grid_part_2.pkl\"\n",
    "CALENDAR = processed_data_dir + \"grid_part_3.pkl\"\n",
    "LAGS = processed_data_dir + \"lags_df_28.pkl\"\n",
    "MEAN_ENC = processed_data_dir + \"mean_encoding_df.pkl\"\n",
    "\n",
    "\n",
    "# SPLITS for lags creation\n",
    "SHIFT_DAY = 28\n",
    "N_LAGS = 15\n",
    "LAGS_SPLIT = [col for col in range(SHIFT_DAY, SHIFT_DAY + N_LAGS)]\n",
    "ROLS_SPLIT = []\n",
    "for i in [1, 7, 14]:\n",
    "    for j in [7, 14, 30, 60]:\n",
    "        ROLS_SPLIT.append([i, j])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train CA_1\n",
      "[LightGBM] [Warning] feature_fraction is set=0.5, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.5\n",
      "[LightGBM] [Warning] min_data_in_leaf is set=4095, min_child_samples=20 will be ignored. Current value: min_data_in_leaf=4095\n",
      "[100]\tvalid_0's rmse: 2.26641\n",
      "[200]\tvalid_0's rmse: 2.05107\n",
      "[300]\tvalid_0's rmse: 2.03155\n",
      "[400]\tvalid_0's rmse: 2.02587\n",
      "[500]\tvalid_0's rmse: 2.02341\n",
      "[600]\tvalid_0's rmse: 2.02109\n",
      "[700]\tvalid_0's rmse: 2.01907\n",
      "[800]\tvalid_0's rmse: 2.01803\n",
      "[900]\tvalid_0's rmse: 2.01586\n",
      "[1000]\tvalid_0's rmse: 2.0143\n",
      "[1100]\tvalid_0's rmse: 2.01334\n",
      "[1200]\tvalid_0's rmse: 2.01265\n",
      "[1300]\tvalid_0's rmse: 2.01178\n",
      "[1400]\tvalid_0's rmse: 2.01159\n",
      "[1500]\tvalid_0's rmse: 2.0108\n",
      "[1600]\tvalid_0's rmse: 2.00956\n",
      "[1700]\tvalid_0's rmse: 2.00804\n",
      "[1800]\tvalid_0's rmse: 2.00788\n",
      "[1900]\tvalid_0's rmse: 2.00753\n",
      "[2000]\tvalid_0's rmse: 2.00711\n",
      "[2100]\tvalid_0's rmse: 2.00638\n",
      "[2200]\tvalid_0's rmse: 2.00593\n",
      "[2300]\tvalid_0's rmse: 2.00511\n",
      "[2400]\tvalid_0's rmse: 2.00471\n",
      "[2500]\tvalid_0's rmse: 2.00401\n",
      "[2600]\tvalid_0's rmse: 2.00324\n",
      "[2700]\tvalid_0's rmse: 2.00265\n",
      "[2800]\tvalid_0's rmse: 2.00235\n",
      "[2900]\tvalid_0's rmse: 2.00219\n",
      "[3000]\tvalid_0's rmse: 2.00185\n",
      "Train CA_2\n",
      "[LightGBM] [Warning] feature_fraction is set=0.5, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.5\n",
      "[LightGBM] [Warning] min_data_in_leaf is set=4095, min_child_samples=20 will be ignored. Current value: min_data_in_leaf=4095\n",
      "[100]\tvalid_0's rmse: 2.17319\n",
      "[200]\tvalid_0's rmse: 1.98373\n",
      "[300]\tvalid_0's rmse: 1.93575\n",
      "[400]\tvalid_0's rmse: 1.91735\n",
      "[500]\tvalid_0's rmse: 1.90807\n",
      "[600]\tvalid_0's rmse: 1.9045\n",
      "[700]\tvalid_0's rmse: 1.90058\n",
      "[800]\tvalid_0's rmse: 1.89835\n",
      "[900]\tvalid_0's rmse: 1.89603\n",
      "[1000]\tvalid_0's rmse: 1.894\n",
      "[1100]\tvalid_0's rmse: 1.89286\n",
      "[1200]\tvalid_0's rmse: 1.8915\n",
      "[1300]\tvalid_0's rmse: 1.89067\n",
      "[1400]\tvalid_0's rmse: 1.889\n",
      "[1500]\tvalid_0's rmse: 1.8888\n",
      "[1600]\tvalid_0's rmse: 1.88795\n",
      "[1700]\tvalid_0's rmse: 1.88759\n",
      "[1800]\tvalid_0's rmse: 1.8871\n",
      "[1900]\tvalid_0's rmse: 1.88629\n",
      "[2000]\tvalid_0's rmse: 1.88569\n",
      "[2100]\tvalid_0's rmse: 1.88491\n",
      "[2200]\tvalid_0's rmse: 1.88435\n",
      "[2300]\tvalid_0's rmse: 1.8843\n",
      "[2400]\tvalid_0's rmse: 1.88355\n",
      "[2500]\tvalid_0's rmse: 1.88267\n",
      "[2600]\tvalid_0's rmse: 1.88221\n",
      "[2700]\tvalid_0's rmse: 1.88182\n",
      "[2800]\tvalid_0's rmse: 1.88138\n",
      "[2900]\tvalid_0's rmse: 1.88127\n",
      "[3000]\tvalid_0's rmse: 1.88118\n",
      "Train CA_3\n",
      "[LightGBM] [Warning] feature_fraction is set=0.5, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.5\n",
      "[LightGBM] [Warning] min_data_in_leaf is set=4095, min_child_samples=20 will be ignored. Current value: min_data_in_leaf=4095\n",
      "[100]\tvalid_0's rmse: 2.79991\n",
      "[200]\tvalid_0's rmse: 2.39198\n",
      "[300]\tvalid_0's rmse: 2.38719\n",
      "[400]\tvalid_0's rmse: 2.37752\n",
      "[500]\tvalid_0's rmse: 2.37394\n",
      "[600]\tvalid_0's rmse: 2.37097\n",
      "[700]\tvalid_0's rmse: 2.36649\n",
      "[800]\tvalid_0's rmse: 2.36351\n",
      "[900]\tvalid_0's rmse: 2.36019\n",
      "[1000]\tvalid_0's rmse: 2.36004\n",
      "[1100]\tvalid_0's rmse: 2.35633\n",
      "[1200]\tvalid_0's rmse: 2.35568\n",
      "[1300]\tvalid_0's rmse: 2.35504\n",
      "[1400]\tvalid_0's rmse: 2.35381\n",
      "[1500]\tvalid_0's rmse: 2.3516\n",
      "[1600]\tvalid_0's rmse: 2.35051\n",
      "[1700]\tvalid_0's rmse: 2.34976\n",
      "[1800]\tvalid_0's rmse: 2.34936\n",
      "[1900]\tvalid_0's rmse: 2.34768\n",
      "[2000]\tvalid_0's rmse: 2.3473\n",
      "[2100]\tvalid_0's rmse: 2.34596\n",
      "[2200]\tvalid_0's rmse: 2.34467\n",
      "[2300]\tvalid_0's rmse: 2.34409\n",
      "[2400]\tvalid_0's rmse: 2.34369\n",
      "[2500]\tvalid_0's rmse: 2.34395\n",
      "[2600]\tvalid_0's rmse: 2.34426\n",
      "[2700]\tvalid_0's rmse: 2.34414\n",
      "[2800]\tvalid_0's rmse: 2.34287\n",
      "[2900]\tvalid_0's rmse: 2.34312\n",
      "[3000]\tvalid_0's rmse: 2.34331\n",
      "Train CA_4\n",
      "[LightGBM] [Warning] feature_fraction is set=0.5, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.5\n",
      "[LightGBM] [Warning] min_data_in_leaf is set=4095, min_child_samples=20 will be ignored. Current value: min_data_in_leaf=4095\n",
      "[100]\tvalid_0's rmse: 1.45323\n",
      "[200]\tvalid_0's rmse: 1.40316\n",
      "[300]\tvalid_0's rmse: 1.40073\n",
      "[400]\tvalid_0's rmse: 1.39854\n",
      "[500]\tvalid_0's rmse: 1.39646\n",
      "[600]\tvalid_0's rmse: 1.39432\n",
      "[700]\tvalid_0's rmse: 1.39294\n",
      "[800]\tvalid_0's rmse: 1.39213\n",
      "[900]\tvalid_0's rmse: 1.39097\n",
      "[1000]\tvalid_0's rmse: 1.39048\n",
      "[1100]\tvalid_0's rmse: 1.38951\n",
      "[1200]\tvalid_0's rmse: 1.3892\n",
      "[1300]\tvalid_0's rmse: 1.38858\n",
      "[1400]\tvalid_0's rmse: 1.38807\n",
      "[1500]\tvalid_0's rmse: 1.38772\n",
      "[1600]\tvalid_0's rmse: 1.38733\n",
      "[1700]\tvalid_0's rmse: 1.38688\n",
      "[1800]\tvalid_0's rmse: 1.38674\n",
      "[1900]\tvalid_0's rmse: 1.38635\n",
      "[2000]\tvalid_0's rmse: 1.38616\n",
      "[2100]\tvalid_0's rmse: 1.38569\n",
      "[2200]\tvalid_0's rmse: 1.38561\n",
      "[2300]\tvalid_0's rmse: 1.38538\n",
      "[2400]\tvalid_0's rmse: 1.38508\n",
      "[2500]\tvalid_0's rmse: 1.38491\n",
      "[2600]\tvalid_0's rmse: 1.38459\n",
      "[2700]\tvalid_0's rmse: 1.3841\n",
      "[2800]\tvalid_0's rmse: 1.38381\n",
      "[2900]\tvalid_0's rmse: 1.38352\n",
      "[3000]\tvalid_0's rmse: 1.38321\n",
      "Train TX_1\n",
      "[LightGBM] [Warning] feature_fraction is set=0.5, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.5\n",
      "[LightGBM] [Warning] min_data_in_leaf is set=4095, min_child_samples=20 will be ignored. Current value: min_data_in_leaf=4095\n",
      "[100]\tvalid_0's rmse: 1.8273\n",
      "[200]\tvalid_0's rmse: 1.64036\n",
      "[300]\tvalid_0's rmse: 1.61972\n",
      "[400]\tvalid_0's rmse: 1.61022\n",
      "[500]\tvalid_0's rmse: 1.60545\n",
      "[600]\tvalid_0's rmse: 1.60217\n",
      "[700]\tvalid_0's rmse: 1.60082\n",
      "[800]\tvalid_0's rmse: 1.59859\n",
      "[900]\tvalid_0's rmse: 1.59808\n",
      "[1000]\tvalid_0's rmse: 1.59725\n",
      "[1100]\tvalid_0's rmse: 1.59646\n",
      "[1200]\tvalid_0's rmse: 1.596\n",
      "[1300]\tvalid_0's rmse: 1.5957\n",
      "[1400]\tvalid_0's rmse: 1.59501\n",
      "[1500]\tvalid_0's rmse: 1.59503\n",
      "[1600]\tvalid_0's rmse: 1.59469\n",
      "[1700]\tvalid_0's rmse: 1.59492\n",
      "[1800]\tvalid_0's rmse: 1.59469\n",
      "[1900]\tvalid_0's rmse: 1.59413\n",
      "[2000]\tvalid_0's rmse: 1.59369\n",
      "[2100]\tvalid_0's rmse: 1.59321\n",
      "[2200]\tvalid_0's rmse: 1.59262\n",
      "[2300]\tvalid_0's rmse: 1.59248\n",
      "[2400]\tvalid_0's rmse: 1.5928\n",
      "[2500]\tvalid_0's rmse: 1.59171\n",
      "[2600]\tvalid_0's rmse: 1.59153\n",
      "[2700]\tvalid_0's rmse: 1.59115\n",
      "[2800]\tvalid_0's rmse: 1.59167\n",
      "[2900]\tvalid_0's rmse: 1.59207\n",
      "[3000]\tvalid_0's rmse: 1.59137\n",
      "Train TX_2\n",
      "[LightGBM] [Warning] feature_fraction is set=0.5, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.5\n",
      "[LightGBM] [Warning] min_data_in_leaf is set=4095, min_child_samples=20 will be ignored. Current value: min_data_in_leaf=4095\n",
      "[100]\tvalid_0's rmse: 2.06068\n",
      "[200]\tvalid_0's rmse: 1.78744\n",
      "[300]\tvalid_0's rmse: 1.7671\n",
      "[400]\tvalid_0's rmse: 1.76378\n",
      "[500]\tvalid_0's rmse: 1.76279\n",
      "[600]\tvalid_0's rmse: 1.76227\n",
      "[700]\tvalid_0's rmse: 1.7622\n",
      "[800]\tvalid_0's rmse: 1.7622\n",
      "[900]\tvalid_0's rmse: 1.76266\n",
      "[1000]\tvalid_0's rmse: 1.76278\n",
      "[1100]\tvalid_0's rmse: 1.76226\n",
      "[1200]\tvalid_0's rmse: 1.76259\n",
      "[1300]\tvalid_0's rmse: 1.76266\n",
      "[1400]\tvalid_0's rmse: 1.76402\n",
      "[1500]\tvalid_0's rmse: 1.76424\n",
      "[1600]\tvalid_0's rmse: 1.76414\n",
      "[1700]\tvalid_0's rmse: 1.764\n",
      "[1800]\tvalid_0's rmse: 1.76433\n",
      "[1900]\tvalid_0's rmse: 1.76434\n",
      "[2000]\tvalid_0's rmse: 1.76447\n",
      "[2100]\tvalid_0's rmse: 1.76452\n",
      "[2200]\tvalid_0's rmse: 1.76443\n",
      "[2300]\tvalid_0's rmse: 1.76461\n",
      "[2400]\tvalid_0's rmse: 1.76478\n",
      "[2500]\tvalid_0's rmse: 1.7649\n",
      "[2600]\tvalid_0's rmse: 1.76456\n",
      "[2700]\tvalid_0's rmse: 1.76389\n",
      "[2800]\tvalid_0's rmse: 1.76374\n",
      "[2900]\tvalid_0's rmse: 1.76373\n",
      "[3000]\tvalid_0's rmse: 1.76336\n",
      "Train TX_3\n",
      "Train WI_1\n",
      "[LightGBM] [Warning] feature_fraction is set=0.5, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.5\n",
      "[LightGBM] [Warning] min_data_in_leaf is set=4095, min_child_samples=20 will be ignored. Current value: min_data_in_leaf=4095\n",
      "[100]\tvalid_0's rmse: 1.74035\n",
      "[200]\tvalid_0's rmse: 1.62768\n",
      "[300]\tvalid_0's rmse: 1.60883\n",
      "[400]\tvalid_0's rmse: 1.60123\n",
      "[500]\tvalid_0's rmse: 1.59775\n",
      "[600]\tvalid_0's rmse: 1.59536\n",
      "[1100]\tvalid_0's rmse: 1.5885\n",
      "[1200]\tvalid_0's rmse: 1.58763\n",
      "[1300]\tvalid_0's rmse: 1.58671\n",
      "[1400]\tvalid_0's rmse: 1.58599\n",
      "[1500]\tvalid_0's rmse: 1.58527\n",
      "[1600]\tvalid_0's rmse: 1.58444\n",
      "[1700]\tvalid_0's rmse: 1.58389\n",
      "[1800]\tvalid_0's rmse: 1.58302\n",
      "[1900]\tvalid_0's rmse: 1.58272\n",
      "[2000]\tvalid_0's rmse: 1.58217\n",
      "[2100]\tvalid_0's rmse: 1.58155\n",
      "[2200]\tvalid_0's rmse: 1.58115\n",
      "[2300]\tvalid_0's rmse: 1.58088\n",
      "[2400]\tvalid_0's rmse: 1.58059\n",
      "[2500]\tvalid_0's rmse: 1.57998\n",
      "[2600]\tvalid_0's rmse: 1.57967\n",
      "[2700]\tvalid_0's rmse: 1.57922\n",
      "[2800]\tvalid_0's rmse: 1.5788\n",
      "[2900]\tvalid_0's rmse: 1.57841\n",
      "[3000]\tvalid_0's rmse: 1.57807\n"
     ]
    }
   ],
   "source": [
    "########################### Train Models\n",
    "#################################################################################\n",
    "from lightgbm import LGBMRegressor\n",
    "from gluonts.model.rotbaum._model import QRX\n",
    "\n",
    "for store_id in STORES_IDS:\n",
    "    print(\"Train\", store_id)\n",
    "\n",
    "    grid_df, features_columns = get_data_by_store(store_id)\n",
    "\n",
    "    train_mask = grid_df[\"d\"] <= END_TRAIN\n",
    "    valid_mask = train_mask & (grid_df[\"d\"] > (END_TRAIN - P_HORIZON))\n",
    "    preds_mask = (grid_df[\"d\"] > (END_TRAIN - 100)) & (\n",
    "        grid_df[\"d\"] <= END_TRAIN + P_HORIZON\n",
    "    )\n",
    "\n",
    "    #    train_data = lgb.Dataset(grid_df[train_mask][features_columns],\n",
    "    #                       label=grid_df[train_mask][TARGET])\n",
    "\n",
    "    #    valid_data = lgb.Dataset(grid_df[valid_mask][features_columns],\n",
    "    #                       label=grid_df[valid_mask][TARGET])\n",
    "\n",
    "    seed_everything(SEED)\n",
    "    estimator = QRX(\n",
    "        model=LGBMRegressor(**lgb_params),  # lgb_wrapper(**lgb_params),\n",
    "        min_bin_size=200,\n",
    "    )\n",
    "    estimator.fit(\n",
    "        grid_df[train_mask][features_columns],\n",
    "        grid_df[train_mask][TARGET],\n",
    "        max_sample_size=1000000,\n",
    "        seed=SEED,\n",
    "        eval_set=(\n",
    "            grid_df[valid_mask][features_columns],\n",
    "            grid_df[valid_mask][TARGET],\n",
    "        ),\n",
    "        verbose=100,\n",
    "        x_train_is_dataframe=True,\n",
    "    )\n",
    "\n",
    "    grid_df = grid_df[preds_mask].reset_index(drop=True)\n",
    "    keep_cols = [col for col in list(grid_df) if \"_tmp_\" not in col]\n",
    "    grid_df = grid_df[keep_cols]\n",
    "\n",
    "    d_sales = grid_df[[\"d\", \"sales\"]]\n",
    "    substitute = d_sales[\"sales\"].values\n",
    "    substitute[(d_sales[\"d\"] > END_TRAIN)] = np.nan\n",
    "    grid_df[\"sales\"] = substitute\n",
    "\n",
    "    grid_df.to_pickle(processed_data_dir + \"test_\" + store_id + \".pkl\")\n",
    "\n",
    "    #    estimator = lgb.train(lgb_params,\n",
    "    #                          train_data,\n",
    "    #                          valid_sets = [valid_data],\n",
    "    #                          verbose_eval = 100,\n",
    "    #                          )\n",
    "\n",
    "    #    display(pd.DataFrame({'name':estimator.feature_name(),\n",
    "    #                          'imp':estimator.feature_importance()}).sort_values('imp',ascending=False).head(25))\n",
    "\n",
    "    model_name = model_dir + \"lgb_model_\" + store_id + \"_v\" + str(VER) + \".bin\"\n",
    "    pickle.dump(estimator, open(model_name, \"wb\"))\n",
    "\n",
    "    del estimator, grid_df, d_sales, substitute  # , train_data, valid_data\n",
    "    gc.collect()\n",
    "\n",
    "    MODEL_FEATURES = features_columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "hide_input": false,
  "kernelspec": {
   "display_name": "conda_mxnet_p36",
   "language": "python",
   "name": "conda_mxnet_p36"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
